#############################################################################
# Creation of tables and figures presented in the article and online appendix
#############################################################################

# Programme:  wpar-analysis01.r
# Date:       4 April 2016
# Author:     Frank Haege, Department of Politics and Administration, University of Limerick
# Contact:		frank.haege@ul.ie

# Description
#############
# This script creates all tables and figures presented in the following article and its online appendix:
# Frank M. H�ge. 
# Political attention in the Council of the European Union: A new dataset of working party meetings, 1995-2014. 
# European Union Politics, forthcoming.


# Remove all objects
rm(list = ls(all = TRUE))

# Load libraries
library(ggplot2)
library(data.table)
library(foreign)
library(grid)

# Set working directory
setwd("E:\\Seagate Sync\\VOL\\Personal folder\\Documents\\Research\\Current projects\\Working party meetings\\Posted\\EUP 160404\\Data analysis")

# Set options
theme_set(theme_bw())
options(stringsAsFactors=F)

# Load data
dates = read.dta("E:\\Seagate Sync\\VOL\\Personal folder\\Documents\\Research\\Current projects\\Working party meetings\\Posted\\EUP 160404\\Output data\\wpar-management09-reshape01a-dates.dta")



###########################
# Generate lists and tables
###########################


# List of working parties by major policy topic (in online appendix)
####################################################################

# Aggregate data by working party
dates = data.table(dates)
wps = dates[, .(dur = sum(dur)), by = list(wpid, wppolicy, wpname, wpbirth, wpdeath)]
wps = data.frame(wps)
dates = data.frame(dates)
wps = wps[order(wps$wppolicy, wps$wpid), c(2, 1, 3:6)]

# Save table as CSV-file
write.csv(wps, file="Tables\\wpar-analysis01-listofworkingparties.csv", na=".", row.names=F)


# Table 1: Working parties and meetings by policy area
######################################################

# Relative and absulute number of working parties
wparties = unique(dates[, 20:21])
wparties = data.table(wparties)
wparties = wparties[, .(freq = .N), by=wppolicy]
wparties = data.frame(wparties)
wparties$perc = round(wparties$freq*100 / sum(wparties$freq), 1)

# Create indicator variable for sub-group
table(nchar(dates$wpid), exclude=NULL)
dates$sub = 0
dates$sub[nchar(dates$wpid)>=4] = 1
table(dates$sub)

# Relative and absulute number of working parties
sparties = unique(dates[dates$sub==1, 20:21])
sparties = data.table(sparties)
sparties = sparties[, .(s.freq = .N), by=wppolicy]
sparties = data.frame(sparties)

# Merge working party data frames
wparties = merge(wparties, sparties, by="wppolicy", all=T)
wparties$s.freq[is.na(wparties$s.freq)] = 0

# Generate other summary variables
wparties$s.perc = round(wparties$s.freq*100 / wparties$freq, 1)
wparties$m.freq = wparties$freq - wparties$s.freq
wparties$m.perc = round(wparties$m.freq*100 / wparties$freq, 1)
wparties$meet.freq = as.vector(table(dates$wppolicy))
wparties$meet.perc = round(wparties$meet.freq*100 / sum(wparties$meet.freq), 1)
wparties$runno = c(1:nrow(wparties))

# Reorder variables
wparties = wparties[, c(1, 6, 7, 4, 5, 2, 3, 8, 9)]

# Calculate totals
totals = data.frame("Total", sum(wparties$m.freq), round(sum(wparties$m.freq)*100 / sum(wparties$freq), 1),
  sum(wparties$s.freq), round(sum(wparties$s.freq)*100 / sum(wparties$freq), 1),
  sum(wparties$freq), round(sum(wparties$perc), 1), sum(wparties$meet.freq), round(sum(wparties$meet.perc), 0))
names(totals) = names(wparties)

# Add totals
wparties = data.frame(lapply(wparties, as.character))
wparties = rbind(wparties, totals)

# Save list as CSV-file
write.csv(wparties, file="Tables\\wpar-analysis01-wpmeetsbypolicy.csv", na=".", row.names=F)


###############################
# Analysis at major topic level
###############################

# Aggregate data by policy area and year
dates = data.table(dates)
dates[, .(dur = sum(dur)), by = list(year, wppolicy)]
meets = dates[, .(dur = sum(dur)), by = list(year, wppolicy)]
meets = meets[, total := sum(dur), by = year]
meets = meets[, poltotal := sum(dur), by = wppolicy]
meets$perc = (meets$dur*100) / meets$total
meets = data.frame(meets)
meets = meets[order(-meets$poltotal, meets$year), ]


# Figure 1: Distribution of attention in the Council of the EU across policy areas, 1995-2014
#############################################################################################

# Boxplot of relative attention
ggplot(meets, aes(x=reorder(wppolicy, poltotal), y=perc)) +
  geom_boxplot() +
  coord_flip() +
  ylab("Relative attention (meeting time in %)") +
  xlab("") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-boxplot.png", width=7.5, height = 5)  


# Figure 1 in online appendix: Distribution of attention in the Council of the EU across policy areas, 1995-2014
################################################################################################################

# Boxplot of absolute attention
ggplot(meets, aes(x=reorder(wppolicy, poltotal), y=dur)) +
  geom_boxplot() +
  coord_flip() +
  ylab("Absolute attention (meeting time in days)") +
  xlab("") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-boxplotabs.png", width=7.5, height = 5)  


# Figure 3: Distribution of attention across time, 1995-2014
############################################################

# Stacked area plot of relative attention
bw.palette = gray.colors(17)
ggplot(meets, aes(x=year,y=perc,fill=reorder(wppolicy, poltotal))) + 
  geom_area() +
  geom_line(position = "stack") +
  scale_fill_manual(values=rev(bw.palette), name="Policy area") + 
  xlab("Year") +
  ylab("Relative attention (meeting time in %)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-stackprop.png", width=7.5, height = 5)  


# Figure 2 in online appendix: Distribution of attention across time, 1995-2014
###############################################################################

# Stacked area plot of absolute attention
bw.palette = gray.colors(17)
ggplot(meets, aes(x=year,y=dur,fill=reorder(wppolicy, poltotal))) + 
  geom_area() +
  geom_line(position = "stack") +
  scale_fill_manual(values=rev(bw.palette), name="Policy area") + 
  xlab("Year") +
  ylab("Absolute attention (meeting time in days)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-stackpropabs.png", width=7.5, height = 5)  


# Figure 4: Attention to policy areas over time, 1995-2014
#########################################################

# Time-series plot of relative attention by policy
ggplot(meets, aes(x=year, y=perc, group=wppolicy)) +
  facet_wrap(~wppolicy, nrow=6, scales="free") +
  geom_point() +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Relative attention (meeting time in %)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-timeseries.png", width=8, height = 10)  


# Figure 3 in online appendix: Attention to policy areas over time, 1995-2014
############################################################################# 

# Time-series plot of absolute attention by policy
ggplot(meets, aes(x=year, y=dur, group=wppolicy)) +
  facet_wrap(~wppolicy, nrow=6, scales="free") +
  geom_point() +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Absolute attention (meeting time in days)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-timeseriesabs.png", width=8, height = 10)  


# Comparison to European Council Data
#####################################

# Read data from http://euagendas.weebly.com/data.html 
# Reference: Petya Alexandrova, Marcello Carammia, Sebastiaan Princen, and Arco Timmermans (2014). 
# Measuring the European Council Agenda: Introducing a New Approach and Dataset. 
# European Union Politics 15(1): 152-167.
ecc = read.csv("E:\\Seagate Sync\\VOL\\Personal folder\\Documents\\Research\\Current projects\\Working party meetings\\Datasets\\ECC Dataset\\ECC_7.1_1975-2014.csv")

# Restrict to time period and statements with policy content
ecc = ecc[ecc$Year>1994, ]
ecc = ecc[ecc$Main.CAPIC!=0, ]

# Aggregate by year
table(ecc$ECPRC, ecc$Year)
table(ecc$ECPRC, ecc$Main.CAPIC)
ecc$one = 1
ecc = data.table(ecc)
ecc.agg = ecc[, .(freq = sum(one)), by=list(Year, Main.CAPIC)]
ecc.agg = ecc.agg[, total := sum(freq), by = Year]
ecc.agg$perc = (ecc.agg$freq*100)/ecc.agg$total
ecc.agg = ecc.agg[, poltotal := sum(freq), by = Main.CAPIC]
ecc.agg = data.frame(ecc.agg)
table(ecc.agg$poltotal, ecc.agg$Main.CAPIC)
table(ecc.agg$ECPRC)

# Code policy variable
ecc.agg$policy = NA
ecc.agg$policy[ecc.agg$Main.CAPIC==1] = "Macroeconomics"
ecc.agg$policy[ecc.agg$Main.CAPIC==2] = "Civil Rights, Minority Issues and Civil Liberties"
ecc.agg$policy[ecc.agg$Main.CAPIC==3] = "Health"
ecc.agg$policy[ecc.agg$Main.CAPIC==4] = "Agriculture and Fisheries"
ecc.agg$policy[ecc.agg$Main.CAPIC==5] = "Labour and Employment"
ecc.agg$policy[ecc.agg$Main.CAPIC==6] = "Education"
ecc.agg$policy[ecc.agg$Main.CAPIC==7] = "Environment"
ecc.agg$policy[ecc.agg$Main.CAPIC==8] = "Energy"
ecc.agg$policy[ecc.agg$Main.CAPIC==9] = "Immigration"
ecc.agg$policy[ecc.agg$Main.CAPIC==10] = "Transportation"
ecc.agg$policy[ecc.agg$Main.CAPIC==12] = "Law and Crime"
ecc.agg$policy[ecc.agg$Main.CAPIC==13] = "Social Policy"
ecc.agg$policy[ecc.agg$Main.CAPIC==14] = "Regional and Urban Policy and Planning"
ecc.agg$policy[ecc.agg$Main.CAPIC==15] = "Banking, Finance and Internal Trade"
ecc.agg$policy[ecc.agg$Main.CAPIC==16] = "Defence"
ecc.agg$policy[ecc.agg$Main.CAPIC==17] = "Space, Science, Technology and Communications"
ecc.agg$policy[ecc.agg$Main.CAPIC==18] = "Foreign Trade"
ecc.agg$policy[ecc.agg$Main.CAPIC==19] = "International Affairs and Foreign Aid"
ecc.agg$policy[ecc.agg$Main.CAPIC==20] = "EU Governance and Government Operations"
ecc.agg$policy[ecc.agg$Main.CAPIC==21] = "Public Lands, Water Management and Territorial Issues"
ecc.agg$policy[ecc.agg$Main.CAPIC==23] = "Culture and Media"


# Figure 2: Distribution of attention in the European Councilacross policy areas, 1995-2014
###########################################################################################

# Boxplot of relative attention
ggplot(ecc.agg, aes(x=reorder(policy, poltotal), y=perc)) +
  geom_boxplot() +
  coord_flip() +
  ylab("Relative attention (quasi-sentences in %)") +
  xlab("") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-boxplot-EC.png", width=7.5, height = 5)  


###########################################################
# Analysis of attention to Immigration and other JHA issues
###########################################################

# Select only Justice and Home Affairs
dates.jha = dates[dates$wppolicy=="Justice & Home Affairs", ]

# Restrict to post-2000 period
dates.jha = dates.jha[dates.jha$year>1999, ]

# Aggregate by semester
dates.jha = data.table(dates.jha)
dates.jha = dates.jha[, .(dur = sum(dur)), by = list(year, semester, wpid, wpname)]
dates.jha = data.frame(dates.jha)

# Generate indicator variable for immigration
dates.jha$immigration = 1
sel.mig = which(dates.jha$wpid=="G01" | dates.jha$wpid=="G02" | dates.jha$wpid=="G03" | dates.jha$wpid=="G04" | dates.jha$wpid=="G04a" | dates.jha$wpid=="G04b" | dates.jha$wpid=="G05" | dates.jha$wpid=="G06" | dates.jha$wpid=="G07")
dates.jha$immigration[sel.mig] = 0
dates.jha$immigration = factor(dates.jha$immigration, labels = c("Immigration", "Other Justice & Home Affairs"))

# Aggregate immigration vs. non-immigration
dates.jha = data.table(dates.jha)
dates.jha = dates.jha[, .(dur = sum(dur)), by = list(year, semester, immigration)]
dates.jha = data.frame(dates.jha)
dates.jha = dates.jha[order(dates.jha$year, dates.jha$immigration, dates.jha$semester), ]
dates.jha$yrsem = dates.jha$year
dates.jha$yrsem[dates.jha$semester=="Second semester"] = dates.jha$yrsem[dates.jha$semester=="Second semester"] + 0.5

# Variable for relative attention
dates.jha = data.table(dates.jha)
dates.jha = dates.jha[, total := sum(dur), by = yrsem]
dates.jha$perc = dates.jha$dur*100 / dates.jha$total
dates.jha = data.frame(dates.jha)


# Figure 5: Attention to Immigration Policy and other Justice & Home Affairs issues, 2000-2014
##############################################################################################

# Line chart of absolute attention
ggplot(dates.jha, aes(x=yrsem, y=dur, group=immigration)) + 
  facet_wrap(~immigration, scales="free") +
  geom_point(size=4, shape=20) +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Absolute attention (meeting time in days)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-jha-tsabs.png", width=7.5, height = 4)  


# Figure 4 in online appendix: Attention to Immigration Policy and other Justice & Home Affairs issues, 2000-2014
#################################################################################################################

# Line chart of relative attention
ggplot(dates.jha, aes(x=yrsem ,y=perc, group=immigration)) + 
  facet_wrap(~immigration, scales="free") +
  geom_point(size=4, shape=20) +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Relative attention (% of JHA meeting time)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-jha-tsrel.png", width=7.5, height = 4)  


##################################################################################################################
# Analysis of attention to Financial Regulation & Fiscal Stability and other Economic and Financial Affairs issues
##################################################################################################################

# Select only Economic and Financial Affairs
dates.efa = dates[dates$wppolicy=="Economic & Financial Affairs", ]

# Restrict to post-2000 period
dates.efa = dates.efa[dates.efa$year>1999, ]

# Aggregate by year
dates.efa = data.table(dates.efa)
dates.efa = dates.efa[, .(dur = sum(dur)), by = list(year, semester, wpid, wpname)]
dates.efa[, sum(dur), by = wpid]
dates.efa = data.frame(dates.efa)
names(table(dates.efa$wpid))
table(dates.efa$wpid, dates.efa$year)

# Generate indicator variable for finance
dates.efa$finance = 0
sel.fin = which(dates.efa$wpid=="A14" | dates.efa$wpid=="F00" | dates.efa$wpid=="F02" | 
                   dates.efa$wpid=="F02a" | dates.efa$wpid=="F05" | grepl("F07", dates.efa$wpid) | 
                   grepl("F07", dates.efa$wpid) | dates.efa$wpid=="F09" | dates.efa$wpid=="F10" | 
                   dates.efa$wpid=="F11" | dates.efa$wpid=="F12" | dates.efa$wpid=="F14" | 
                   dates.efa$wpid=="F15" | dates.efa$wpid=="F16" | grepl("F08", dates.efa$wpid))
dates.efa$finance[sel.fin] = 1
table(dates.efa$wpname[dates.efa$finance==0])
table(dates.efa$wpname[dates.efa$finance==1])
dates.efa$finance = factor(dates.efa$finance, labels = c("Financial Regulation & Fiscal Stability", "Other Economic & Financial Affairs"))

# Aggregate finance vs. non-finance
dates.efa = data.table(dates.efa)
dates.efa = dates.efa[, .(dur = sum(dur)), by = list(year, semester, finance)]
dates.efa = data.frame(dates.efa)
dates.efa = dates.efa[order(dates.efa$year, dates.efa$finance), ]
dates.efa$yrsem = dates.efa$year
dates.efa$yrsem[dates.efa$semester=="Second semester"] = dates.efa$yrsem[dates.efa$semester=="Second semester"] + 0.5

# Variable for relative attention
dates.efa = data.table(dates.efa)
dates.efa = dates.efa[, total := sum(dur), by = yrsem]
dates.efa$perc = dates.efa$dur*100 / dates.efa$total
dates.efa = data.frame(dates.efa)


# Figure 6: Attention to Financial Regulation & Fiscal Stability and other Economic & Financial Affairs issues, 2000-2014
#########################################################################################################################

# Line chart of absolute attention
ggplot(dates.efa, aes(x=yrsem, y=dur, group=finance)) + 
  facet_wrap(~finance, scales="free") +
  geom_point(size=4, shape=20) +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Absolute attention (meeting time in days)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-efa-tsabs.png", width=7.5, height = 4)  


# Figure 5 in online appendix: Attention to Financial Regulation & Fiscal Stability and other Economic & Financial Affairs issues, 2000-2014
############################################################################################################################################

# Line chart of relative attention
ggplot(dates.efa, aes(x=year ,y=perc, group=finance)) + 
  facet_wrap(~finance, scales="free") +
  geom_point(size=4, shape=20) +
  geom_smooth(method="loess", colour="black") +
  xlab("Year") +
  ylab("Relative attention (% of EcoFin meeting time)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-efa-tsrel.png", width=7.5, height = 4)  


#####################################
# Analysis of attention to CFP reform
#####################################

# Select only Fisheries policy
dates.fish = dates[dates$wppolicy=="Fisheries", ]

# Restrict to WP on Internal Fisheries Policy
dates.fish = dates.fish[dates.fish$wpid=="I02", ]
dates.fish = dates.fish[dates.fish$year>2010 | (dates.fish$year==2010 & dates.fish$semester=="Second semester"), ]


# Figure 7: Attention to Fisheries Policy during negotiations on the Common Fisheries Policy reform, 1 July 2010 to 31 December 2014
####################################################################################################################################

# Time-series plot of absolute monthly attention by working party
ggplot(dates.fish, aes(x=date, y=dur)) +
  geom_point(size=6, shape=20, alpha=0.3, position = position_jitter(w = 0, h = 0.02)) +
  scale_y_continuous(breaks=seq(0.5,1,0.1)) +
  geom_text(data = NULL, x = 15050, y = 0.70, label = "Commission\nproposal", size=4) +
  geom_segment(data = NULL, x = 15050, xend = 15169, y = 0.67, yend = 0.67, arrow=arrow(angle=10, type="closed")) +
  geom_vline(xintercept=15169, linetype="dashed") +
  geom_text(data = NULL, x = 15300, y = 0.70, label = "Council\ndiscussions", size=4) +
  geom_segment(data = NULL, x = 15300, xend = 15474, y = 0.67, yend = 0.67, arrow=arrow(angle=10, type="closed")) +
  geom_segment(data = NULL, x = 15300, xend = 15457, y = 0.67, yend = 0.65, arrow=arrow(angle=10, type="closed")) +
  geom_segment(data = NULL, x = 15300, xend = 15418, y = 0.67, yend = 0.63, arrow=arrow(angle=10, type="closed")) +
  geom_vline(xintercept=15418) +                             # 19-03-2012
  geom_vline(xintercept=15457) +                             # 27-04-2012
  geom_vline(xintercept=15474) +                             # 14-05-2012
  geom_vline(xintercept=15503) +                             # 12-06-2012
  geom_text(data = NULL, x = 15625, y = 0.70, label = "Council\nagreement", size=4) +
  geom_segment(data = NULL, x = 15625, xend = 15503, y = 0.67, yend = 0.67, arrow=arrow(angle=10, type="closed")) +
  geom_text(data = NULL, x = 15623, y = 0.63, label = "Council\ndiscussions", size=4) +
  geom_segment(data = NULL, x = 15625, xend = 15817, y = 0.60, yend = 0.60, arrow=arrow(angle=10, type="closed")) +
  geom_segment(data = NULL, x = 15625, xend = 15733, y = 0.60, yend = 0.585, arrow=arrow(angle=10, type="closed")) +
  geom_vline(xintercept=15733) +                             # 28-01-2013
  geom_vline(xintercept=15817) +                             # 22-04-2013
  geom_vline(xintercept=15901) +                             # 15-07-2013
  geom_vline(xintercept=15996, linetype="dashed") +          # 18-10-2013
  geom_text(data = NULL, x = 16200, y = 0.63, label = "Council agreement\nwith EP", size=4) +
  geom_segment(data = NULL, x = 16200, xend = 15901, y = 0.60, yend = 0.60, arrow=arrow(angle=10, type="closed")) +
  geom_text(data = NULL, x = 16200, y = 0.57, label = "Council adoption\nCommon Position", size=4) +
  geom_segment(data = NULL, x = 16200, xend = 15996, y = 0.54, yend = 0.54, arrow=arrow(angle=10, type="closed")) +
  geom_smooth(method="loess", span=0.65, colour="black") +
  xlab("Time") +
  ylab("Absolute attention (meeting time in days)") +
  theme(text = element_text(size = 12))
ggsave("Figures\\wpar-analysis01-fish-tsabs.png", width=7.5, height = 6)  


